#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Datetime: 2021/10/05 14:17
# @Author  : CHEN Wang
# @Site    :
# @File    : crypto_timing_factor_batch_test.py
# @Software: PyCharm

"""
脚本说明： 数字货币择时指标批量测试脚本
"""

import os
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pylab import mpl
from quant_researcher.quant.project_tool import hammer
from quant_researcher.quant.project_tool.localize import TEST_DIR, DATA_DIR, TA_TEST_DIR
from quant_researcher.quant.datasource_fetch.crypto_api.glassnode import get_prices, get_ret, all_http, get_indicators
from quant_researcher.quant.datasource_fetch.crypto_api import sanbase
from quant_researcher.quant.datasource_fetch.crypto_api.self_defined import onchain_metrics_list, trading_metrics_list, social_metrics_list
from quant_researcher.quant.factors.indicator_tools.alpha101.Alpha101 import alpha101_name_list
from quant_researcher.quant.project_tool.logger.my_logger import LOG
from quant_researcher.quant.factors.factor_analysis.factor_analyser.crypto_timing_factor_analyse import backtest_test, ta_indic_backtest_test
from quant_researcher.quant.factors.factor_preprocess.preprocess import de_noise_all_method
from quant_researcher.quant.factors.indicator_tools.indicators_util import ta_func_list, ta_func_util_dict

mpl.rcParams['font.sans-serif'] = ['SimHei']  # 设置字体为黑体
mpl.rcParams['axes.unicode_minus'] = False  # 解决中文字体负号显示不正常问题


def get_already_tested_indic():
    """
    检测哪些指标已经经过全面分析，返回已经测试过的指标列表

    """
    # 完整的指标测试必须包含指标名_origin, 指标名_diff_d, 指标名_diff_w, 指标名_diff_m, 指标名五个文件夹， 且diff_m包含backtest_result_all.xlsx
    self_defined_metrics = onchain_metrics_list + trading_metrics_list + social_metrics_list
    social_metrics = sanbase.get_social_metrics_list(asset='bitcoin')
    # 所有指标
    all_factor_list = list(all_http.keys()) + self_defined_metrics + social_metrics + alpha101_name_list

    already_tested_indic = []
    for factor_name in all_factor_list:
        file_path1 = os.path.join(TEST_DIR, f'{factor_name}')
        path_name = factor_name + f'_origin'
        file_path2 = os.path.join(TEST_DIR, f'{path_name}')
        path_name = factor_name + f'_diff_d'
        file_path3 = os.path.join(TEST_DIR, f'{path_name}')
        path_name = factor_name + f'_diff_w'
        file_path4 = os.path.join(TEST_DIR, f'{path_name}')
        path_name = factor_name + f'_diff_m'
        file_path5 = os.path.join(TEST_DIR, f'{path_name}')
        file_name5 = os.path.join(file_path5, f'backtest_result_all.xlsx')
        if os.path.exists(file_path1) and os.path.exists(file_path2) and os.path.exists(file_path3) \
                and os.path.exists(file_path4) and os.path.exists(file_path5) and os.path.exists(f'{file_name5}'):
            already_tested_indic.append(factor_name)
    already_tested_indic.sort()
    return already_tested_indic


def get_already_tested_ta_indic():
    """
    检测哪些技术指标已经经过全面分析，返回已经测试过的技术指标列表

    """
    # 完整的技术指标测试，指标名文件夹下必须包含backtest_result_all.xlsx

    already_tested_ta_indic = []
    for func in ta_func_list:
        file_path1 = os.path.join(TA_TEST_DIR, f'{func.__name__}')
        file_name1 = os.path.join(file_path1, f'backtest_result_all.xlsx')
        if os.path.exists(file_path1) and os.path.exists(f'{file_name1}'):
            already_tested_ta_indic.append(func)
    return already_tested_ta_indic


def factor_batch_test(factor_name, **kwargs):
    asset = kwargs.pop('asset')
    start_date = kwargs.pop('start_date')
    end_date = kwargs.pop('end_date')
    prices_df = kwargs.pop('prices_df')
    asset_ret_series = kwargs.pop('asset_ret_series')
    all_asset_ret_df = kwargs.pop('all_asset_ret_df')
    benchmark_ret_df = kwargs.pop('benchmark_ret_df')
    commission = kwargs.pop('commission', 0.001)

    self_defined_metrics = onchain_metrics_list + trading_metrics_list + social_metrics_list
    social_metrics = sanbase.get_social_metrics_list(asset='bitcoin')
    if factor_name in self_defined_metrics:
        if factor_name in onchain_metrics_list:
            file_path1 = os.path.join(DATA_DIR, f'onchain_data')
        elif factor_name in trading_metrics_list:
            file_path1 = os.path.join(DATA_DIR, f'trading_data')
        elif factor_name in social_metrics_list:
            file_path1 = os.path.join(DATA_DIR, f'social_data')
        else:
            raise NotImplementedError
        file_name1 = os.path.join(file_path1, f'{factor_name}')
        origin_factor_df = pd.read_csv(f'{file_name1}.csv', index_col='end_date')
        origin_factor_df = origin_factor_df.loc[start_date:end_date, :]
    elif factor_name in social_metrics:
        # origin_factor_df = sanbase.get_indicators(indic_name=factor_name, asset=asset, start_date=start_date, end_date=end_date)
        file_path1 = os.path.join(DATA_DIR, f'social_data')
        file_name1 = os.path.join(file_path1, f'{factor_name}')
        origin_factor_df = pd.read_csv(f'{file_name1}.csv', index_col='end_date')
        origin_factor_df = origin_factor_df.loc[start_date:end_date, :]
    else:
        try:
            origin_factor_df = get_indicators(indic_name=factor_name, asset=asset, start_date=start_date,
                                              end_date=end_date)
        except Exception as e:
            if '无法成功链接' in str(e):
                LOG.info(f'{asset}对应的{factor_name}数据无法成功链接，休息60秒，重新尝试连接')
                time.sleep(60)  # 休息60秒，然后重新连接
                try:
                    origin_factor_df = get_indicators(indic_name=factor_name, asset=asset, start_date=start_date,
                                                      end_date=end_date)
                except Exception as e:
                    LOG.info(f'{asset}对应的{factor_name}数据无法成功链接，{str(e)}, 跳过该指标')
                    return
            else:  # 出现其他异常
                LOG.info(f'{asset}对应的{factor_name}数据获取出现异常，{str(e)}, 跳过该指标')
                return

    if origin_factor_df is None:
        LOG.info(f'{asset}对应的{factor_name}数据无法获取')
        return
    if len(origin_factor_df.columns) > 1:
        LOG.info(f'{factor_name}数据包含多列，需要进一步指定')
        return
    if isinstance(origin_factor_df.values[0][0], dict) or isinstance(origin_factor_df.values[0][0], str):
        # 指标值既不是float，也不是int，直接跳过
        LOG.info(f'{factor_name}的指标值是dict 或者 str, 不是数值, 请检查')
        return
    if origin_factor_df.values[0][0] > 1000000000000000:  # 如果数据过大进行缩放
        LOG.info(f'{asset}对应的{factor_name}数据过大，进行了缩放，直接除以1000000000000000')
        origin_factor_df = origin_factor_df / 1000000000000000

    file_path = os.path.join(TEST_DIR, f'{factor_name}')
    os.makedirs(file_path, exist_ok=True)
    file_name = os.path.join(file_path, f'{factor_name}')
    origin_factor_df.to_csv(f'{file_name}.csv')  # 数据保存

    origin_factor_series = origin_factor_df[factor_name]
    factor_end_date = origin_factor_series.index[-1]
    if end_date > factor_end_date:  # 因为部分因子取不到最新数据，则回测到指标最新时间
        backtest_end_date = factor_end_date
    else:
        backtest_end_date = end_date

    origin_factor_series_1 = origin_factor_series.copy()
    origin_factor_series_1.index = pd.to_datetime(origin_factor_series_1.index)
    log_price = prices_df['log_prices']
    log_price.index = pd.to_datetime(log_price.index)
    asset_prices_series = prices_df['close']

    fig = plt.figure(figsize=(8, 6))
    ax1 = fig.add_subplot(111)
    # prices_df['close'].plot(ax=ax1, style='bD--', alpha=0.4, label='比特币价格')  # alpha表示点的透明程度
    log_price.plot(ax=ax1, style='b', alpha=0.4, label=f'{asset}对数价格')  # alpha表示点的透明程度
    # plt.xticks(np.arange(1990,2016))
    plt.xlabel('日期')
    # ax1.set_yticks(np.arange(0,0.9,0.1))        # 设置左边纵坐标刻度
    # ax1.set_ylabel('价格')  # 设置左边纵坐标标签
    plt.legend(loc=2)  # 设置图例在左上方

    ax2 = ax1.twinx()
    origin_factor_series_1.plot(ax=ax2, grid=True, label=f'{factor_name}', style='y', alpha=0.7)
    # ax2.set_yticks(np.arange(0,0.121,0.015))    # 设置右边纵坐标刻度
    ax2.set_ylabel(f'{factor_name}')  # 设置右边纵坐标标签
    plt.legend(loc=1)  # 设置图例在右上方
    plt.title(f'{factor_name} & {asset} log_price')  # 给整张图命名

    file_name = os.path.join(file_path, f'{factor_name}-{asset} log_price')
    plt.savefig(f'{file_name}.png', dpi=400, bbox_inches='tight')
    # plt.show()  # 图片不用展示
    fig.clf()
    plt.close(fig)

    # origin_factor_series.index = pd.to_datetime(origin_factor_series.index)
    # de_noise_all_method(origin_factor_series, ts_freq='D', lamda=1000, if_plot=True, period=7, min_periods=7, span=7)

    origin_factor_series = origin_factor_series.fillna(method='ffill')  # 指标中间缺失的需要往后填充
    all_perf = backtest_test(asset, None, backtest_end_date, origin_factor_series, asset_prices_series,
                             asset_ret_series, all_asset_ret_df, benchmark_ret_df, commission=commission)
    if all_perf.empty:
        return False
    else:
        return True


def ta_factor_batch_test(ta_func, **kwargs):
    """
    技术分析指标批量测试

    :param ta_func_name: 技术分析函数对象
    :param kwargs:
    :return:
    """

    if ta_func.__name__ in ta_func_util_dict.keys():
        asset = kwargs.pop('asset')
        start_date = kwargs.pop('start_date')
        end_date = kwargs.pop('end_date')
        ohlcv_data = kwargs.pop('ohlcv_data')
        prices_df = kwargs.pop('prices_df')
        asset_ret_series = kwargs.pop('asset_ret_series')
        all_asset_ret_df = kwargs.pop('all_asset_ret_df')
        benchmark_ret_df = kwargs.pop('benchmark_ret_df')
        commission = kwargs.pop('commission', 0.001)

        file_path = os.path.join(TA_TEST_DIR, f'{ta_func.__name__}')
        os.makedirs(file_path, exist_ok=True)
        print(f'开始测试技术指标函数{ta_func.__name__}')
        all_perf = ta_indic_backtest_test(asset, start_date, end_date, ta_func, ohlcv_data,
                                          prices_df, asset_ret_series,
                                          all_asset_ret_df, benchmark_ret_df, commission=commission)
        print(f'技术指标函数{ta_func.__name__}测试结束')
        if all_perf.empty:
            return False
        else:
            return True
    else:
        print('该技术指标没有说明取值范围和指定适用的使用方法，请重新输入')
        return False


if __name__ == '__main__':
    asset = 'BTC'
    start_date = '2015-01-01'
    # start_date = '2018-06-01'
    end_date = '2021-11-13'
    # end_date = '2022-02-21'

    file_name = os.path.join(DATA_DIR, f'{asset}_price_log_price')
    prices_df = pd.read_csv(f'{file_name}.csv', index_col=0)
    file_name = os.path.join(DATA_DIR, f'{asset}_USDT_ret')
    all_asset_ret = pd.read_csv(f'{file_name}.csv', index_col=0)

    asset_ret_series = all_asset_ret[asset]  # 提取该资产收益率数据，进行后续回测胜率检验
    all_asset_ret_df = all_asset_ret.unstack().reset_index()
    all_asset_ret_df.columns = ['code', 'tradedate', 'daily_ret']  # 对该资产以及USDT收益率数据变形，满足回测需要
    benchmark_ret_df = pd.DataFrame(all_asset_ret[asset])  # 提取该资产收益率数据，作为后续回测检验的基准

    # %% 测试一般指标
    self_defined_metrics = onchain_metrics_list + trading_metrics_list + social_metrics_list
    social_metrics = sanbase.get_social_metrics_list(asset='bitcoin')
    # 所有指标
    all_factor_list = list(all_http.keys()) + self_defined_metrics + social_metrics

    # 测试尚未测试过的指标
    already_tested_indic = get_already_tested_indic()
    factor_list = list(set(all_factor_list) - set(already_tested_indic))

    # 测试指定指标
    # factor_list = ['aSOPR']
    # factor_list = ['Entities Supply Distribution']
    # factor_list = ['close_ma140_ratio', 'close_ma120_ratio']
    # factor_list = ['Illiquid Supply Shock', 'Long-Term Holder Supply Shock']
    # factor_list = [i for i in factor_list if ('Futures Perpetual Funding Rate' in i)]
    # factor_list = [i for i in factor_list if ('Stablecoin Supply Ratio (SSR)' in i)]

    # # 重新测试精选指标
    # file_name = os.path.join(TEST_DIR, f'all_selected_strategy_signal_method')
    # factor_list = list(pd.read_excel(f'{file_name}.xlsx')['factor_name'])  # 数据保存

    # # 附加测试指标
    # factor_list.extend(self_defined_metrics)

    # 批处理框架
    # info = {
    #     'func': factor_batch_test,
    #     'asset': asset,
    #     'start_date': start_date,
    #     'end_date': end_date,
    #     'prices_df': prices_df,
    #     'asset_ret_series': asset_ret_series,
    #     'all_asset_ret_df': all_asset_ret_df,
    #     'benchmark_ret_df': benchmark_ret_df,
    # }
    #
    # hammer.split_a_list_into_batches_and_deal_each_with_a_process(
    #     the_list=factor_list, info=info,
    #     ignore_err=True, how_many=16,
    #     what_version='v2'
    # )

    # # 循环跑
    # for factor_name in factor_list:
    #     factor_batch_test(factor_name, asset=asset, start_date=start_date, end_date=end_date, prices_df=prices_df,
    #                       asset_ret_series=asset_ret_series, all_asset_ret_df=all_asset_ret_df,
    #                       benchmark_ret_df=benchmark_ret_df)

    # # %% 测试技术指标
    file_name = os.path.join(DATA_DIR, f'BTC_history_ohlcvm')
    all_data = pd.read_excel(f'{file_name}.xlsx', index_col='end_date')
    all_data = all_data.replace(0, None)  # 数据为空的需要处理

    # 测试全部技术指标
    ta_func_list_for_test = ta_func_list.copy()

    # # 测试尚未测试过的指标
    already_tested_ta_indic = get_already_tested_ta_indic()
    ta_func_list_for_test = list(set(ta_func_list) - set(already_tested_ta_indic))

    # 测试指定指标
    # from quant_researcher.quant.factors.indicator_tools.indicators import indicator_ASI
    # ta_func_list_for_test = [indicator_ASI]

    # 批处理框架
    info = {
        'func': ta_factor_batch_test,
        'asset': asset,
        'start_date': start_date,
        'end_date': end_date,
        'ohlcv_data': all_data,
        'prices_df': prices_df,
        'asset_ret_series': asset_ret_series,
        'all_asset_ret_df': all_asset_ret_df,
        'benchmark_ret_df': benchmark_ret_df,
    }

    hammer.split_a_list_into_batches_and_deal_each_with_a_process(
        the_list=ta_func_list_for_test, info=info,
        ignore_err=True, how_many=60,
        what_version='v2'
    )


    # 循环跑
    # for ta_func in ta_func_list_for_test:
    #     ta_factor_batch_test(ta_func, asset=asset, start_date=start_date, end_date=end_date, ohlcv_data=all_data,
    #                          prices_df=prices_df, asset_ret_series=asset_ret_series, all_asset_ret_df=all_asset_ret_df,
    #                          benchmark_ret_df=benchmark_ret_df)